package com.andeveloper.stockinfo;

import java.io.IOException;
import java.net.InetAddress;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.HashMap;
import java.util.Map;

import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.InetSocketTransportAddress;
import org.elasticsearch.transport.client.PreBuiltTransportClient;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;



public class StockInfoCollector {
	
	public static void main(String[] args) throws IOException, ParseException {
		TransportClient client = new PreBuiltTransportClient(Settings.EMPTY).addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName("localhost"), 9300));
		Document doc = Jsoup.connect("https://gupiao.baidu.com/").get();
		Element news = doc.select("div.mix-news").first();
		Elements links = news.getElementsByTag("a");
		for (Element link : links) {
		  String linkHref = link.attr("abs:href");
		  String newsTitle = link.text();
		  Document article = Jsoup.connect(linkHref).get();
		  String newsTime = article.select("div.news-header").first().getElementsByTag("span").first().text();
		  Element newsBox = article.select("div.news-box").first();
		  String newsContent = newsBox.html().replaceAll("<.*?>", "");
		  Map<String, Object> map = new HashMap<String, Object>();
		  map.put("newsTitle", newsTitle);
		  map.put("newsContent", newsContent);
		  SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
		  map.put("newsTime", simpleDateFormat.parse(newsTime));
		  client.prepareIndex("stock", "article").setSource(map).get();
		}
	}
}
